
# 데이터 분석
import pandas as pd
import numpy as np
import random as rnd
import csv
from collections import Counter

# 시각화
import seaborn as sns
import plotly.express as px
import matplotlib.pyplot as plt
%matplotlib inline

# 기계 학습
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC, LinearSVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.linear_model import Perceptron
from sklearn.linear_model import SGDClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.neural_network import MLPClassifier


#데이터 불러오기
transac_data = pd.read_csv('transactions.csv')
train_data = pd.read_csv('train.csv')
test_data = pd.read_csv('test.csv')
songs_data = pd.read_csv('songs.csv')


transac_data.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 1819855 entries, 0 to 1819854
Data columns (total 6 columns):
 #   Column      Dtype 
---  ------      ----- 
 0   user_id     int64 
 1   song_id     int64 
 2   rec_loc     object
 3   rec_screen  object
 4   entry       object
 5   listen      int64 
dtypes: int64(3), object(3)
memory usage: 83.3+ MB


transac_data.head()


transac_data['user_id'] = transac_data['user_id'].astype(object)
transac_data['song_id'] = transac_data['song_id'].astype(object)
transac_data['listen'] = transac_data['listen'].astype(object)


rec_data = transac_data['song_id'].value_counts()
rec_data

98      7254
38      7085
266     6865
150     6741
412     6409
        ... 
8781      48
9049      47
7880      45
7421      45
8535      40
Name: song_id, Length: 4878, dtype: int64


#가장 추천이 많이 된 곡
songs_data[songs_data['song_id'] == 98]


#가장 추천이 적게 된 곡
songs_data[songs_data['song_id'] == 8535]


# 추천 성공 / 실패 트랜잭션
success = transac_data[transac_data['listen'] == 1]
fail= transac_data[transac_data['listen'] == 0]


rec_success = success['song_id'].value_counts()
rec_success

38      5981
98      5976
266     5490
2       5121
150     5100
        ... 
9189      11
7867      11
1878      11
4975       8
9387       7
Name: song_id, Length: 4878, dtype: int64


#추천 성공률
rec_rate = pd.merge(rec_data, rec_success, left_index=True, right_index=True, how='left')
rec_rate['rec_rate'] = round(rec_rate['song_id_y'] / rec_rate['song_id_x'] *100,2)
rec_rate.sort_values(by=['rec_rate'], axis=0,inplace=True, ascending=False)
rec_rate.columns = ['all', 'suc', 'rec_rate']
rec_rate


#추천 성공률이 높은 곡
songs_data[songs_data['song_id'] == 431]


songs_data.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 4878 entries, 0 to 4877
Data columns (total 8 columns):
 #   Column    Non-Null Count  Dtype 
---  ------    --------------  ----- 
 0   song_id   4878 non-null   int64 
 1   length    4878 non-null   int64 
 2   genre     4878 non-null   int64 
 3   artist    4878 non-null   int64 
 4   composer  4878 non-null   int64 
 5   lyricist  4878 non-null   int64 
 6   language  4878 non-null   int64 
 7   isrc      4878 non-null   object
dtypes: int64(7), object(1)
memory usage: 305.0+ KB


songs_data.head()


#1) object로 형변환(genre / artist / composer / lyricist / language)
songs_data['song_id'] = songs_data['song_id'].astype(object)
songs_data['genre'] = songs_data['genre'].astype(object)
songs_data['artist'] = songs_data['artist'].astype(object)
songs_data['composer'] = songs_data['composer'].astype(object)
songs_data['lyricist'] = songs_data['lyricist'].astype(object)
songs_data['language'] = songs_data['language'].astype(object)


#형변환 완료 확인
songs_data.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 4878 entries, 0 to 4877
Data columns (total 8 columns):
 #   Column    Non-Null Count  Dtype 
---  ------    --------------  ----- 
 0   song_id   4878 non-null   object
 1   length    4878 non-null   int64 
 2   genre     4878 non-null   object
 3   artist    4878 non-null   object
 4   composer  4878 non-null   object
 5   lyricist  4878 non-null   object
 6   language  4878 non-null   object
 7   isrc      4878 non-null   object
dtypes: int64(1), object(7)
memory usage: 305.0+ KB


train_data.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 7000 entries, 0 to 6999
Data columns (total 7 columns):
 #   Column      Non-Null Count  Dtype 
---  ------      --------------  ----- 
 0   user_id     7000 non-null   int64 
 1   city        7000 non-null   int64 
 2   age         7000 non-null   int64 
 3   reg_method  7000 non-null   int64 
 4   reg_date    7000 non-null   int64 
 5   ex_date     7000 non-null   int64 
 6   gender      7000 non-null   object
dtypes: int64(6), object(1)
memory usage: 382.9+ KB


train_data.head()


#1) object로 형변환(city / reg_method)
train_data['city'] = train_data['city'].astype(object)
train_data['reg_method'] = train_data['reg_method'].astype(object)

#2) datetime으로 형변환(reg_date / ex_date)
train_data['reg_date'] = train_data['reg_date'].astype(str)
train_data['reg_date'] = train_data['reg_date'].str[0:4] + "-" +train_data['reg_date'].str[4:6] + "-" + train_data['reg_date'].str[6:8]
train_data['reg_date'] = train_data['reg_date'].astype('datetime64[ns]')

train_data['ex_date'] = train_data['ex_date'].astype(str)
train_data['ex_date'] = train_data['ex_date'].str[0:4] + "-" +train_data['ex_date'].str[4:6] + "-" + train_data['ex_date'].str[6:8]
train_data['ex_date'] = train_data['ex_date'].astype('datetime64[ns]')


#형변환 완료 확인
train_data.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 7000 entries, 0 to 6999
Data columns (total 7 columns):
 #   Column      Non-Null Count  Dtype         
---  ------      --------------  -----         
 0   user_id     7000 non-null   int64         
 1   city        7000 non-null   object        
 2   age         7000 non-null   int64         
 3   reg_method  7000 non-null   object        
 4   reg_date    7000 non-null   datetime64[ns]
 5   ex_date     7000 non-null   datetime64[ns]
 6   gender      7000 non-null   object        
dtypes: datetime64[ns](2), int64(2), object(3)
memory usage: 382.9+ KB


test_data.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 1679 entries, 0 to 1678
Data columns (total 7 columns):
 #   Column      Non-Null Count  Dtype  
---  ------      --------------  -----  
 0   user_id     1679 non-null   int64  
 1   city        1679 non-null   int64  
 2   age         1679 non-null   int64  
 3   reg_method  1679 non-null   int64  
 4   reg_date    1679 non-null   int64  
 5   ex_date     1679 non-null   int64  
 6   gender      0 non-null      float64
dtypes: float64(1), int64(6)
memory usage: 91.9 KB


test_data.head()


#1) object로 형변환(city / reg_method)
test_data['city'] = test_data['city'].astype(object)
test_data['reg_method'] = test_data['reg_method'].astype(object)

#2) datetime으로 형변환(reg_date / ex_date)
test_data['reg_date'] = test_data['reg_date'].astype(str)
test_data['reg_date'] = test_data['reg_date'].str[0:4] + "-" +test_data['reg_date'].str[4:6] + "-" + test_data['reg_date'].str[6:8]
test_data['reg_date'] = test_data['reg_date'].astype('datetime64[ns]')

test_data['ex_date'] = test_data['ex_date'].astype(str)
test_data['ex_date'] = test_data['ex_date'].str[0:4] + "-" +test_data['ex_date'].str[4:6] + "-" + test_data['ex_date'].str[6:8]
test_data['ex_date'] = test_data['ex_date'].astype('datetime64[ns]')


#형변환 완료 확인
test_data.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 1679 entries, 0 to 1678
Data columns (total 7 columns):
 #   Column      Non-Null Count  Dtype         
---  ------      --------------  -----         
 0   user_id     1679 non-null   int64         
 1   city        1679 non-null   object        
 2   age         1679 non-null   int64         
 3   reg_method  1679 non-null   object        
 4   reg_date    1679 non-null   datetime64[ns]
 5   ex_date     1679 non-null   datetime64[ns]
 6   gender      0 non-null      float64       
dtypes: datetime64[ns](2), float64(1), int64(2), object(2)
memory usage: 91.9+ KB


train_data.describe()


train_data['count']=1
train_data['gender'].value_counts()

male      3600
female    3400
Name: gender, dtype: int64


# train data 남녀 분포
fig = px.pie(train_data, values='count', names='gender')
fig.show()


# train data 나이 분포
sns.countplot(x='age',data=train_data, palette='Set2')
plt.show()


#이상값 age가 0살이거나 110세 이상인 경우
age_outlier = train_data[(train_data['age'] < 1) | (train_data['age'] > 110) ]
age_outlier


#이상값 age>110 확인
train_data[train_data['age'] > 110]


#age 이상값 성별분포 확인
age_outlier.groupby('gender').count()


def func_age(row):
    if row['age'] < 1 or row['age'] > 110:
        return 27
    else:
        return row['age']
train_data['age'] = train_data.apply(func_age, axis=1)


#이상값 대체 확인
train_data[(train_data['age'] < 1) | (train_data['age'] > 110) ]


# train data 나이 분포 다시 확인
sns.countplot(x='age',data=train_data, palette='Set2')
plt.show()


transac_data.head()


transac_data.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 1819855 entries, 0 to 1819854
Data columns (total 6 columns):
 #   Column      Dtype 
---  ------      ----- 
 0   user_id     object
 1   song_id     object
 2   rec_loc     object
 3   rec_screen  object
 4   entry       object
 5   listen      object
dtypes: object(6)
memory usage: 83.3+ MB


transac_data['count']=1
transac_data


# transac_data rec_loc 분포
transac_data['rec_loc'].value_counts()

my library      929302
discover        559231
search          131825
listen with      86494
radio            73250
explore          38104
notification      1440
settings           209
Name: rec_loc, dtype: int64


# transac_data rec_screen 분포
transac_data['rec_screen'].value_counts()

Local playlist more     873111
Online playlist more    378137
Others profile more      85748
Discover Chart           75455
Radio                    74030
Discover Feature         65963
Album more               63203
Artist more              62534
Search                   55473
My library               20032
Explore                  19634
Unknown                  17920
Discover Genre           11808
Discover New              5474
Search Home               5036
Search Trends             4283
My library_Search         1965
Self profile more           39
Concert                      6
Payment                      4
Name: rec_screen, dtype: int64


# transac_data entry 분포
transac_data['entry'].value_counts()

local-library             585531
online-playlist           550462
local-playlist            302692
listen-with                80370
radio                      74491
album                      68861
top-hits-for-artist        63781
song                       47309
song-based-playlist        42481
topic-article-playlist      3411
artist                       399
my-daily-playlist             67
Name: entry, dtype: int64


#m_transac_train = pd.merge(transac_data, train_data, on="user_id", how='left')
m_transac_train = pd.merge(transac_data, train_data, on="user_id", how='inner')
#중복된 count 속성 삭제 후 재추가
m_transac_train.drop('count_x', axis=1, inplace=True)
m_transac_train.drop('count_y', axis=1, inplace=True)
m_transac_train['count']=1
m_transac_train.info()

<class 'pandas.core.frame.DataFrame'>
Int64Index: 1019674 entries, 0 to 1019673
Data columns (total 13 columns):
 #   Column      Non-Null Count    Dtype         
---  ------      --------------    -----         
 0   user_id     1019674 non-null  object        
 1   song_id     1019674 non-null  object        
 2   rec_loc     1019674 non-null  object        
 3   rec_screen  1019674 non-null  object        
 4   entry       1019674 non-null  object        
 5   listen      1019674 non-null  object        
 6   city        1019674 non-null  object        
 7   age         1019674 non-null  int64         
 8   reg_method  1019674 non-null  object        
 9   reg_date    1019674 non-null  datetime64[ns]
 10  ex_date     1019674 non-null  datetime64[ns]
 11  gender      1019674 non-null  object        
 12  count       1019674 non-null  int64         
dtypes: datetime64[ns](2), int64(2), object(9)
memory usage: 108.9+ MB


m_transac_train


m_transac_train['gender'].value_counts()

male      526089
female    493585
Name: gender, dtype: int64


songs_data.head()


mm_transac_train = pd.merge(m_transac_train, songs_data, on="song_id")


#중복된 count 속성 삭제 후 재추가
mm_transac_train.drop('count', axis=1, inplace=True)
mm_transac_train['count']=1


mm_transac_train.info()

<class 'pandas.core.frame.DataFrame'>
Int64Index: 1019674 entries, 0 to 1019673
Data columns (total 20 columns):
 #   Column      Non-Null Count    Dtype         
---  ------      --------------    -----         
 0   user_id     1019674 non-null  object        
 1   song_id     1019674 non-null  object        
 2   rec_loc     1019674 non-null  object        
 3   rec_screen  1019674 non-null  object        
 4   entry       1019674 non-null  object        
 5   listen      1019674 non-null  object        
 6   city        1019674 non-null  object        
 7   age         1019674 non-null  int64         
 8   reg_method  1019674 non-null  object        
 9   reg_date    1019674 non-null  datetime64[ns]
 10  ex_date     1019674 non-null  datetime64[ns]
 11  gender      1019674 non-null  object        
 12  length      1019674 non-null  int64         
 13  genre       1019674 non-null  object        
 14  artist      1019674 non-null  object        
 15  composer    1019674 non-null  object        
 16  lyricist    1019674 non-null  object        
 17  language    1019674 non-null  object        
 18  isrc        1019674 non-null  object        
 19  count       1019674 non-null  int64         
dtypes: datetime64[ns](2), int64(3), object(15)
memory usage: 163.4+ MB


# 형변환
mm_transac_train['user_id'] = mm_transac_train['user_id'].astype(object)
mm_transac_train['song_id'] = mm_transac_train['song_id'].astype(object)
mm_transac_train['listen'] = mm_transac_train['listen'].astype(object)


df = mm_transac_train


df.describe()


#성별에 따른 장르 분포
fig = px.sunburst(df, path=['gender', 'genre'], values='count')
fig.show()

df


df['isrc'] = df['isrc'].astype(str)
df['country_code'] = df['isrc'].str[0:2]
df['first_regist_code'] = df['isrc'].str[2:5]
df['year_of_referance'] = df['isrc'].str[5:7]
df['designation_code'] = df['isrc'].str[7:]
df


df['country_code'].value_counts()

TW    791964
HK     53213
US     46064
KR     29764
CN     25503
GB     22605
JP     11133
NO      9231
MY      4557
NL      3942
SG      3941
SE      3315
DE      3046
GT      2553
TG      1797
CA      1508
DK      1504
QM      1276
EM       927
MO       462
AU       359
FR       280
UK       233
NG       191
TC       131
CY       101
GL        40
TH        34
Name: country_code, dtype: int64


df['gender'].unique()

array(['male', 'female'], dtype=object)


agg_format={'age':'mean', 'length':'mean'}
df.groupby('gender').agg(agg_format)


agg_format={'age':'median', 'length':'median'}
df.groupby('gender').agg(agg_format)


#음원 발매 나라별 청취 도시 비율
fig = px.sunburst(df, path=['country_code', 'city'], values='count')
fig.show()


#청취 device 통계 생성
device = transac_data.groupby(['rec_loc', 'rec_screen','entry','listen'])['user_id'].count()
device.to_csv('device.csv')


def func_year(row):
    if int(row['year_of_referance']) > 20:
        return int(row['year_of_referance']) + 1900
    else:
        return int(row['year_of_referance']) + 2000
df['year_of_referance'] = df.apply(func_year, axis=1)


len(df['year_of_referance'].unique())

37


year_dist = df.groupby('year_of_referance').mean()
year_dist = pd.DataFrame(year_dist)
year_dist
# fig = px.line(year_dist, x="year_of_referance", y="n_age")
# fig.show()


fig = px.box(df, y="age")
fig.show()


year_dist = year_dist.reset_index()


#발매 연도별 청취연령 boxplot
fig = px.box(df, x="year_of_referance", y="age", color="year_of_referance")
fig.show()


#발매 연도별 청취연령 평균(점점 어려짐)
fig = px.scatter(year_dist, x="year_of_referance", y="age")
fig.show()


new_songs_data=songs_data
new_songs_data['isrc'] = new_songs_data['isrc'].astype(str)
new_songs_data['country_code'] = new_songs_data['isrc'].str[0:2]
new_songs_data['first_regist_code'] = new_songs_data['isrc'].str[2:5]
new_songs_data['year_of_referance'] = new_songs_data['isrc'].str[5:7]
new_songs_data['designation_code'] = new_songs_data['isrc'].str[7:]
new_songs_data


new_songs_data['year_of_referance'] = new_songs_data.apply(func_year, axis=1)
new_songs_data


new_songs_data_des = new_songs_data.groupby('year_of_referance').mean()
new_songs_data_des = new_songs_data_des.reset_index()


#발매 연도별 노래 길이 boxplot
fig = px.box(new_songs_data, x="year_of_referance", y="length", color="year_of_referance")
fig.show()


#노래들의 평균 길이가 점점 짧아짐
fig = px.scatter(new_songs_data_des, x="year_of_referance", y="length")
fig.show()


songs_data[songs_data['year_of_referance']==1986]


songs_data[songs_data['year_of_referance']==1980]


songs_data['year_of_referance'].value_counts()

2016    1220
2015     538
2014     509
2013     365
2012     306
2011     230
2009     201
2010     175
2008     166
2006     153
2007     147
2005     119
2000     111
2004     103
2001     101
2003      93
2002      87
1999      64
2017      52
1998      36
1997      28
1996      19
1993      10
1995      10
1992       6
1994       5
1986       4
1990       4
1989       4
1988       4
1987       2
1984       1
1927       1
1991       1
1980       1
1985       1
1968       1
Name: year_of_referance, dtype: int64


df.to_csv('transac_train_song_merge.csv',index = False)


df_open = pd.read_csv('transac_train_song_merge.csv')
df_open


df_open.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 1019674 entries, 0 to 1019673
Data columns (total 24 columns):
 #   Column             Non-Null Count    Dtype 
---  ------             --------------    ----- 
 0   user_id            1019674 non-null  int64 
 1   song_id            1019674 non-null  int64 
 2   rec_loc            1019674 non-null  object
 3   rec_screen         1019674 non-null  object
 4   entry              1019674 non-null  object
 5   listen             1019674 non-null  int64 
 6   city               1019674 non-null  int64 
 7   age                1019674 non-null  int64 
 8   reg_method         1019674 non-null  int64 
 9   reg_date           1019674 non-null  object
 10  ex_date            1019674 non-null  object
 11  gender             1019674 non-null  object
 12  length             1019674 non-null  int64 
 13  genre              1019674 non-null  int64 
 14  artist             1019674 non-null  int64 
 15  composer           1019674 non-null  int64 
 16  lyricist           1019674 non-null  int64 
 17  language           1019674 non-null  int64 
 18  isrc               1019674 non-null  object
 19  count              1019674 non-null  int64 
 20  country_code       1019674 non-null  object
 21  first_regist_code  1019674 non-null  object
 22  year_of_referance  1019674 non-null  int64 
 23  designation_code   1019674 non-null  int64 
dtypes: int64(15), object(9)
memory usage: 186.7+ MB


df_open.drop(['user_id'], axis='columns', inplace=True)
df_open.drop(['song_id'], axis='columns', inplace=True)
df_open.drop(['entry'], axis='columns', inplace=True)
df_open.drop(['city'], axis='columns', inplace=True)
df_open.drop(['reg_method'], axis='columns', inplace=True)
df_open.drop(['reg_date'], axis='columns', inplace=True)
df_open.drop(['ex_date'], axis='columns', inplace=True)
df_open.drop(['artist'], axis='columns', inplace=True)
df_open.drop(['composer'], axis='columns', inplace=True)
df_open.drop(['lyricist'], axis='columns', inplace=True)
df_open.drop(['language'], axis='columns', inplace=True)
df_open.drop(['isrc'], axis='columns', inplace=True)
df_open.drop(['count'], axis='columns', inplace=True)
df_open.drop(['first_regist_code'], axis='columns', inplace=True)
df_open.drop(['year_of_referance'], axis='columns', inplace=True)
df_open.drop(['designation_code'], axis='columns', inplace=True)
df_open.drop(['rec_screen'], axis='columns', inplace=True)
df_open.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 1019674 entries, 0 to 1019673
Data columns (total 7 columns):
 #   Column        Non-Null Count    Dtype 
---  ------        --------------    ----- 
 0   rec_loc       1019674 non-null  object
 1   listen        1019674 non-null  int64 
 2   age           1019674 non-null  int64 
 3   gender        1019674 non-null  object
 4   length        1019674 non-null  int64 
 5   genre         1019674 non-null  int64 
 6   country_code  1019674 non-null  object
dtypes: int64(4), object(3)
memory usage: 54.5+ MB


df_open2 = pd.get_dummies(df_open, columns=["rec_loc"])
df_open3 = pd.get_dummies(df_open2, columns=["country_code"])


df_pre = pd.get_dummies(df_open3, columns=["gender"])


df_pre.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 1019674 entries, 0 to 1019673
Data columns (total 42 columns):
 #   Column                Non-Null Count    Dtype
---  ------                --------------    -----
 0   listen                1019674 non-null  int64
 1   age                   1019674 non-null  int64
 2   length                1019674 non-null  int64
 3   genre                 1019674 non-null  int64
 4   rec_loc_discover      1019674 non-null  uint8
 5   rec_loc_explore       1019674 non-null  uint8
 6   rec_loc_listen with   1019674 non-null  uint8
 7   rec_loc_my library    1019674 non-null  uint8
 8   rec_loc_notification  1019674 non-null  uint8
 9   rec_loc_radio         1019674 non-null  uint8
 10  rec_loc_search        1019674 non-null  uint8
 11  rec_loc_settings      1019674 non-null  uint8
 12  country_code_AU       1019674 non-null  uint8
 13  country_code_CA       1019674 non-null  uint8
 14  country_code_CN       1019674 non-null  uint8
 15  country_code_CY       1019674 non-null  uint8
 16  country_code_DE       1019674 non-null  uint8
 17  country_code_DK       1019674 non-null  uint8
 18  country_code_EM       1019674 non-null  uint8
 19  country_code_FR       1019674 non-null  uint8
 20  country_code_GB       1019674 non-null  uint8
 21  country_code_GL       1019674 non-null  uint8
 22  country_code_GT       1019674 non-null  uint8
 23  country_code_HK       1019674 non-null  uint8
 24  country_code_JP       1019674 non-null  uint8
 25  country_code_KR       1019674 non-null  uint8
 26  country_code_MO       1019674 non-null  uint8
 27  country_code_MY       1019674 non-null  uint8
 28  country_code_NG       1019674 non-null  uint8
 29  country_code_NL       1019674 non-null  uint8
 30  country_code_NO       1019674 non-null  uint8
 31  country_code_QM       1019674 non-null  uint8
 32  country_code_SE       1019674 non-null  uint8
 33  country_code_SG       1019674 non-null  uint8
 34  country_code_TC       1019674 non-null  uint8
 35  country_code_TG       1019674 non-null  uint8
 36  country_code_TH       1019674 non-null  uint8
 37  country_code_TW       1019674 non-null  uint8
 38  country_code_UK       1019674 non-null  uint8
 39  country_code_US       1019674 non-null  uint8
 40  gender_female         1019674 non-null  uint8
 41  gender_male           1019674 non-null  uint8
dtypes: int64(4), uint8(38)
memory usage: 68.1 MB


df_pre.drop(['gender_male'], axis='columns', inplace=True)


df_pre.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 1019674 entries, 0 to 1019673
Data columns (total 41 columns):
 #   Column                Non-Null Count    Dtype
---  ------                --------------    -----
 0   listen                1019674 non-null  int64
 1   age                   1019674 non-null  int64
 2   length                1019674 non-null  int64
 3   genre                 1019674 non-null  int64
 4   rec_loc_discover      1019674 non-null  uint8
 5   rec_loc_explore       1019674 non-null  uint8
 6   rec_loc_listen with   1019674 non-null  uint8
 7   rec_loc_my library    1019674 non-null  uint8
 8   rec_loc_notification  1019674 non-null  uint8
 9   rec_loc_radio         1019674 non-null  uint8
 10  rec_loc_search        1019674 non-null  uint8
 11  rec_loc_settings      1019674 non-null  uint8
 12  country_code_AU       1019674 non-null  uint8
 13  country_code_CA       1019674 non-null  uint8
 14  country_code_CN       1019674 non-null  uint8
 15  country_code_CY       1019674 non-null  uint8
 16  country_code_DE       1019674 non-null  uint8
 17  country_code_DK       1019674 non-null  uint8
 18  country_code_EM       1019674 non-null  uint8
 19  country_code_FR       1019674 non-null  uint8
 20  country_code_GB       1019674 non-null  uint8
 21  country_code_GL       1019674 non-null  uint8
 22  country_code_GT       1019674 non-null  uint8
 23  country_code_HK       1019674 non-null  uint8
 24  country_code_JP       1019674 non-null  uint8
 25  country_code_KR       1019674 non-null  uint8
 26  country_code_MO       1019674 non-null  uint8
 27  country_code_MY       1019674 non-null  uint8
 28  country_code_NG       1019674 non-null  uint8
 29  country_code_NL       1019674 non-null  uint8
 30  country_code_NO       1019674 non-null  uint8
 31  country_code_QM       1019674 non-null  uint8
 32  country_code_SE       1019674 non-null  uint8
 33  country_code_SG       1019674 non-null  uint8
 34  country_code_TC       1019674 non-null  uint8
 35  country_code_TG       1019674 non-null  uint8
 36  country_code_TH       1019674 non-null  uint8
 37  country_code_TW       1019674 non-null  uint8
 38  country_code_UK       1019674 non-null  uint8
 39  country_code_US       1019674 non-null  uint8
 40  gender_female         1019674 non-null  uint8
dtypes: int64(4), uint8(37)
memory usage: 67.1 MB


from sklearn.model_selection import train_test_split
# 데이터 내에서 학습 집합과 테스트 집합을 나눔(8:2)
df_train, df_test = train_test_split(df_pre, test_size=0.2)


X_df_train = df_train.drop("gender_female", axis=1)
Y_df_train = df_train["gender_female"]
X_df_test = df_test.drop("gender_female", axis=1)
Y_df_test = df_test["gender_female"]
X_df_train.shape, Y_df_train.shape, X_df_test.shape, Y_df_test.shape

((815739, 40), (815739,), (203935, 40), (203935,))


# SVM 모델 학습
svc = SVC()
svc.fit(X_df_train, Y_df_train)


# 테스트 데이터에 대해 예측
Y_pred_svc = svc.predict(X_df_test)
acc_svc = round(svc.score(Y_pred_svc, Y_test) * 100, 2) #소수점 2자리까지(반올림해줌)
acc_svc


# 테스트 데이터에 대해 예측
Y_pred_svc = svc.predict(X_test)
acc_svc = round(svc.score(Y_pred_svc, Y_test) * 100, 2) #소수점 2자리까지(반올림해줌)
acc_svc


# Logistic Regression training
logreg = LogisticRegression()
logreg.fit(X_train, Y_train)


# Logistic Regression prediction
Y_pred_logreg = logreg.predict(X_test) 
acc_log = round(logreg.score(Y_pred_logreg, Y_test)*100,2)
acc_log


# 속성별 회귀 계수
coeff_df  =  pd.DataFrame(pre_train_data.columns.delete(0))
coeff_df.columns = ['Feature']
coeff_df["Correlation"] = pd.Series(logreg.coef_[0])

coeff_df = coeff_df.sort_values(by='Correlation', ascending=False)
coeff_df


knn  = KNeighborsClassifier(n_neighbors = 3)
knn.fit(X_train, Y_train)
Y_pred_knn = knn.predict(X_test)
acc_knn = round(knn.score(X_test, Y_test) * 100, 2) #소수점 2자리까지(반올림해줌)
acc_knn


# Gaussian Naive Bayes
gaussian = GaussianNB()
gaussian.fit(X_train, Y_train)
Y_pred_NB = gaussian.predict(X_test)
acc_gaussian = round(gaussian.score(Y_pred_NB, Y_test)*100,2)
acc_gaussian


decision_tree = DecisionTreeClassifier()
decision_tree.fit(X_train,Y_train)
Y_pred_DT = decision_tree.predict(X_test)
acc_decision_tree = round(decision_tree.score(Y_pred_DT, Y_test)*100,2)
acc_decision_tree


ANN = MLPClassifier(solver='lbfgs', alpha=1, hidden_layer_sizes=(20,10), random_state=1)
ANN.fit(X_train, Y_train)
Y_pred_ANN = ANN.predict(X_test)
acc_ANN = round(ANN.score(Y_pred_ANN, Y_test) * 100, 2)
acc_ANN


#precision / recall / f1 score
from sklearn.metrics import recall_score, precision_score, f1_score


# target 0 = 남자
print(round(precision_score(Y_test, Y_pred_logreg)*100,2))
print(round(recall_score(Y_test, Y_pred_logreg)*100,2))
print(round(f1_score(Y_test, Y_pred_logreg)*100,2))
# target 0 = 여자
Y_T = np.where(Y_test==0, 1, 0)
Y_P = np.where(Y_pred_logreg==0, 1, 0)
print(round(precision_score(Y_T, Y_P)*100,2))
print(round(recall_score(Y_T, Y_P)*100,2))
print(round(f1_score(Y_T, Y_P)*100,2))


train_data


def top_1(candidates):
    """리스트 중 최빈값 추출"""
    assert isinstance(candidates, list), 'Must be a list type'
    if len(candidates) == 0: return None
    return Counter(candidates).most_common(n=1)[0][0]


def mode_list(id_list, feature):
    return_list = []
    for i in id_list:
        return_list.append(top_1(list(transac_data[transac_data['user_id']==i][feature])))
        print(len(return_list))
    return return_list


train_data['song_id'] = mode_list(list(train_data['user_id']),'song_id')


train_data['rec_loc'] = mode_list(list(train_data['user_id']),'rec_loc')
train_data['rec_screen'] = mode_list(list(train_data['user_id']),'rec_screen')
train_data['entry'] = mode_list(list(train_data['user_id']),'entry')
train_data['listen'] = mode_list(list(train_data['user_id']),'listen')
train_data.to_csv('pre_train_data.csv')


train_data.info()


pre_train_data = pd.read_csv('pre_train_data.csv')
pre_train_data.drop(['Unnamed: 0'], axis='columns', inplace=True)


#transaction 없는 데이터 13개 있음
pre_train_data.isna().sum()


#user가 가장 많이 들은 음악의 장르, 언어
#transac user 중심으로 작성
#12,379명의 유저, 4,878곡
# transac_data['user_id'].value_counts()
# transac_data['song_id'].value_counts()
sorted(transac_data['user_id'].unique())


m_transac_songs = pd.merge(transac_data, songs_data, on="song_id")


#발매나라와 발매곡의 언어는 관계가 있으나 완전일치는 아님
#언어 중 31은 한국어, 52는 영어, 3은 중국어, 17은 일본어 등을 유추할 수 있음
m_transac_songs.groupby(['country_code','language'])['user_id','song_id'].count()


m_transac_songs.groupby(['language','country_code'])['user_id','song_id'].count()


m_transac_songs.info()


year_max = m_transac_songs.groupby('user_id')['year_of_referance'].max().reset_index()
year_min = m_transac_songs.groupby('user_id')['year_of_referance'].min().reset_index()
length_mean = m_transac_songs.groupby('user_id')['length'].mean().reset_index()
tran_count = m_transac_songs.groupby('user_id')['count'].sum().reset_index()
tran_count


pre_train_data


pre_train_data = pd.merge(pre_train_data, year_max, on="user_id")
pre_train_data = pd.merge(pre_train_data, year_min, on="user_id")
pre_train_data = pd.merge(pre_train_data, length_mean, on="user_id")
pre_train_data = pd.merge(pre_train_data, tran_count, on="user_id")
pre_train_data


pre_train_data.drop('count_x', axis=1, inplace=True)


# 필요없는 열 제거
pre_train_data.drop('reg_date', axis=1, inplace=True)
pre_train_data.drop('ex_date', axis=1, inplace=True)


# pre_train_data = pd.merge(pre_train_data, songs_data, on="song_id")
# pre_train_data


transac_data = m_transac_songs


pre_train_data2 = pre_train_data


pre_train_data2['genre'] = mode_list(list(pre_train_data2['user_id']),'genre')
pre_train_data2['language'] = mode_list(list(pre_train_data2['user_id']),'language')
pre_train_data2['country_code'] = mode_list(list(pre_train_data2['user_id']),'country_code')
pre_train_data2['artist'] = mode_list(list(pre_train_data2['user_id']),'artist')
pre_train_data2.to_csv('pre_train_data_add.csv')


pre_train_data2 = pd.read_csv('pre_train_data_add.csv')
pre_train_data2


pre_train_data2.drop(['Unnamed: 0'], axis='columns', inplace=True)
pre_train_data2.drop(['count_x'], axis='columns', inplace=True)
pre_train_data2


pre_train_data2['range']=pre_train_data2['year_of_referance_x']-pre_train_data2['year_of_referance_y']


pre_train_data = pre_train_data2


pre_train_data


# 필요없는 열 제거
pre_train_data.drop('user_id', axis=1, inplace=True)
pre_train_data.drop('reg_date', axis=1, inplace=True)
pre_train_data.drop('ex_date', axis=1, inplace=True)


# dummy변수 처리 (pre_train_data)
pre_train_data = pd.get_dummies(pre_train_data, columns=["city"])
pre_train_data = pd.get_dummies(pre_train_data, columns=["reg_method"])
pre_train_data = pd.get_dummies(pre_train_data, columns=["song_id"])
pre_train_data = pd.get_dummies(pre_train_data, columns=["rec_loc"])
pre_train_data = pd.get_dummies(pre_train_data, columns=["rec_screen"])
pre_train_data = pd.get_dummies(pre_train_data, columns=["entry"])
pre_train_data = pd.get_dummies(pre_train_data, columns=["genre"])
pre_train_data = pd.get_dummies(pre_train_data, columns=["language"])
pre_train_data = pd.get_dummies(pre_train_data, columns=["country_code"])
pre_train_data = pd.get_dummies(pre_train_data, columns=["artist"])


pre_train_data.info()


#성별처리
pre_train_data = pd.get_dummies(pre_train_data, columns=["gender"])


#성별은 하나만 남김
pre_train_data.drop('gender_male', axis=1, inplace=True)


pre_train_data.info()


a = list(pre_train_data.columns)
a[5] = 'count'
pre_train_data.columns = a
print(a)


test_data['song_id'] = mode_list(list(test_data['user_id']),'song_id')
test_data['rec_loc'] = mode_list(list(test_data['user_id']),'rec_loc')
test_data['rec_screen'] = mode_list(list(test_data['user_id']),'rec_screen')
test_data['entry'] = mode_list(list(test_data['user_id']),'entry')
test_data['listen'] = mode_list(list(test_data['user_id']),'listen')


test_data.to_csv('pre_test_data.csv')


pre_test_data = pd.read_csv('pre_test_data.csv')


pre_test_data = pd.merge(pre_test_data, year_max, on="user_id")
pre_test_data = pd.merge(pre_test_data, year_min, on="user_id")
pre_test_data = pd.merge(pre_test_data, length_mean, on="user_id")
pre_test_data = pd.merge(pre_test_data, tran_count, on="user_id")


pre_test_data


pre_test_data['genre'] = mode_list(list(pre_test_data['user_id']),'genre')
pre_test_data['language'] = mode_list(list(pre_test_data['user_id']),'language')
pre_test_data['country_code'] = mode_list(list(pre_test_data['user_id']),'country_code')
pre_test_data['artist'] = mode_list(list(pre_test_data['user_id']),'artist')


pre_test_data.info()


pre_test_data.drop('Unnamed: 0', axis=1, inplace=True)


pre_test_data['range']=pre_test_data['year_of_referance_x']-pre_test_data['year_of_referance_y']
pre_test_data


# 필요없는 열 제거
pre_test_data.drop('user_id', axis=1, inplace=True)
pre_test_data.drop('reg_date', axis=1, inplace=True)
pre_test_data.drop('ex_date', axis=1, inplace=True)
pre_test_data.info()


# dummy변수 처리 (pre_test_data)
pre_test_data = pd.get_dummies(pre_test_data, columns=["city"])
pre_test_data = pd.get_dummies(pre_test_data, columns=["reg_method"])
pre_test_data = pd.get_dummies(pre_test_data, columns=["song_id"])
pre_test_data = pd.get_dummies(pre_test_data, columns=["rec_loc"])
pre_test_data = pd.get_dummies(pre_test_data, columns=["rec_screen"])
pre_test_data = pd.get_dummies(pre_test_data, columns=["entry"])
pre_test_data = pd.get_dummies(pre_test_data, columns=["genre"])
pre_test_data = pd.get_dummies(pre_test_data, columns=["language"])
pre_test_data = pd.get_dummies(pre_test_data, columns=["country_code"])
pre_test_data = pd.get_dummies(pre_test_data, columns=["artist"])


#성별처리(일단 모두 0으로)
pre_test_data.drop('gender', axis=1, inplace=True)
pre_test_data["gender"] = 0


pre_test_data


pre_train_data.info()


pre_test_data.info()


pre_train_data.to_csv('preprocess_train_data.csv')
pre_test_data.to_csv('preprocess_test_data.csv')


#데이터 불러오기
preprocess_train_data = pd.read_csv('preprocess_train_data.csv')
preprocess_train_data.drop(['Unnamed: 0'], axis='columns', inplace=True)
preprocess_test_data = pd.read_csv('preprocess_test_data.csv')
preprocess_test_data.drop(['Unnamed: 0'], axis='columns', inplace=True)


preprocess_train_data


# 학습데이터 내에서 학습 집합과 테스트 집합을 나눔(8:2)
train, test = train_test_split(preprocess_train_data, test_size=0.2)


X_train = train.drop("gender_female", axis=1)
Y_train = train["gender_female"]
X_test = test.drop("gender_female", axis=1)
Y_test = test["gender_female"]
X_train.shape, Y_train.shape, X_test.shape, Y_test.shape


# SVM 모델 학습
svc = SVC()
svc.fit(X_train, Y_train)


# 테스트 데이터에 대해 예측
Y_pred_svc = svc.predict(X_test)
acc_svc = round(svc.score(X_test, Y_test) * 100, 2) #소수점 2자리까지(반올림해줌)
acc_svc


# 테스트 데이터에 대해 예측
Y_pred_svc = svc.predict(X_test)
acc_svc = round(svc.score(X_test, Y_test) * 100, 2) #소수점 2자리까지(반올림해줌)
acc_svc


# Logistic Regression training
logreg = LogisticRegression()
logreg.fit(X_train, Y_train)


# Logistic Regression prediction
Y_pred_logreg = logreg.predict(X_test) 
acc_log = round(logreg.score(X_test, Y_test)*100,2)
acc_log


# Logistic Regression prediction
Y_pred_logreg = logreg.predict(X_test) 
acc_log = round(logreg.score(X_test, Y_test)*100,2)
acc_log


# 속성별 회귀 계수
coeff_df  =  pd.DataFrame(pre_train_data.columns.delete(0))
coeff_df.columns = ['Feature']
coeff_df["Correlation"] = pd.Series(logreg.coef_[0])

coeff_df = coeff_df.sort_values(by='Correlation', ascending=False)
coeff_df


knn  = KNeighborsClassifier(n_neighbors = 3)
knn.fit(X_train, Y_train)
Y_pred_knn = knn.predict(X_test)
acc_knn = round(knn.score(X_test, Y_test) * 100, 2) #소수점 2자리까지(반올림해줌)
acc_knn


# Gaussian Naive Bayes
gaussian = GaussianNB()
gaussian.fit(X_train, Y_train)
Y_pred_NB = gaussian.predict(X_test)
acc_gaussian = round(gaussian.score(X_test, Y_test)*100,2)
acc_gaussian


decision_tree = DecisionTreeClassifier()
decision_tree.fit(X_train,Y_train)
Y_pred_DT = decision_tree.predict(X_test)
acc_decision_tree = round(decision_tree.score(X_test, Y_test)*100,2)
acc_decision_tree


ANN = MLPClassifier(solver='lbfgs', alpha=1, hidden_layer_sizes=(20,10), random_state=1)
ANN.fit(X_train, Y_train)
Y_pred_ANN = ANN.predict(X_test)
acc_ANN = round(ANN.score(X_test, Y_test) * 100, 2)
acc_ANN


#precision / recall / f1 score
from sklearn.metrics import recall_score, precision_score, f1_score


# target 0 = 남자
print(round(precision_score(Y_test, Y_pred_logreg)*100,2))
print(round(recall_score(Y_test, Y_pred_logreg)*100,2))
print(round(f1_score(Y_test, Y_pred_logreg)*100,2))
# target 0 = 여자
Y_T = np.where(Y_test==0, 1, 0)
Y_P = np.where(Y_pred_logreg==0, 1, 0)
print(round(precision_score(Y_T, Y_P)*100,2))
print(round(recall_score(Y_T, Y_P)*100,2))
print(round(f1_score(Y_T, Y_P)*100,2))


from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.preprocessing import StandardScaler
feature = preprocess_train_data.drop("gender_female", axis=1)
scaled_preprocess_train_data = StandardScaler().fit_transform(feature)
lda = LinearDiscriminantAnalysis(n_components=2)
lda.fit(scaled_preprocess_train_data, preprocess_train_data['gender_female'])


X_train_pca.shape, Y_train.shape, X_test.shape, Y_test.shape


pca = PCA(n_components=10)
pca.fit(X_test)
X_test_pca = pca.transform(X_test)
X_test_pca = pd.DataFrame(X_train_pca,columns=[1,2,3,4,5,6,7,8,9,10])
X_test_pca

	song_id	length	genre	artist	composer	lyricist	language	isrc
0	5981	273554	465	330	1787	1533	3	TWC950206108
1	5767	228623	465	633	484	1057	3	TWA531398021
2	8128	257602	458	777	1916	2115	3	TWAY31508002
3	1992	199157	458	548	1011	1904	3	TWA471596001
4	2665	246015	465	671	1407	1362	3	TWK231306302

	user_id	city	age	reg_method	reg_date	ex_date	gender
0	16169	13	21	3	20120118	20180604	NaN
1	4694	13	0	3	20150110	20171014	NaN
2	10654	1	18	3	20150402	20180312	NaN
3	9327	14	16	4	20160824	20171002	NaN
4	3631	22	26	9	20100927	20171225	NaN

	user_id	age
count	7000.000000	7000.000000
mean	11160.084571	28.128857
std	6904.768964	14.844875
min	1.000000	0.000000
25%	5224.250000	22.000000
50%	10691.500000	27.000000
75%	16693.000000	33.000000
max	24977.000000	931.000000

	user_id	city	age	reg_method	reg_date	ex_date	gender	count
24	22786	1	0	3	2014-01-16	2017-06-11	female	1
122	16328	18	0	3	2015-02-15	2017-09-17	female	1
149	18779	5	0	7	2013-10-07	2017-11-25	female	1
151	1115	4	0	4	2016-06-15	2017-09-26	female	1
153	21698	4	0	9	2004-03-28	2017-09-26	female	1
...	...	...	...	...	...	...	...	...
6842	17981	5	0	9	2015-03-22	2017-08-09	female	1
6915	21227	1	0	3	2015-02-05	2016-11-29	female	1
6969	16355	13	0	9	2005-10-09	2017-09-30	male	1
6978	5877	22	0	9	2015-01-28	2017-09-14	male	1
6989	22281	1	0	7	2010-10-05	2017-04-20	female	1

	song_id	length	genre	artist	composer	lyricist	language	isrc
0	5981	273554	465	330	1787	1533	3	TWC950206108
1	5767	228623	465	633	484	1057	3	TWA531398021
2	8128	257602	458	777	1916	2115	3	TWAY31508002
3	1992	199157	458	548	1011	1904	3	TWA471596001
4	2665	246015	465	671	1407	1362	3	TWK231306302

<ml 교육 과제1> 남녀 성별 분류 모델¶

데이터 로딩 및 확인¶

transac_data¶

1) object로 형변환(user_id / song_id / listen)¶

songs_data¶

1) object로 형변환(song_id / genre / artist / composer / lyricist / language)¶

train_data¶

1) object로 형변환(city / reg_method)
2) datetime으로 형변환(reg_date / ex_date)

test_data¶

1) object로 형변환(city / reg_method)
2) datetime으로 형변환(reg_date / ex_date)

Training Data 탐색¶

age 이상값 대체(중앙값 27로)¶

만 0세는 있을 수 있으나, 나이 분포 고려했을 떄 이상값의 비율이 높을 것으로 예상됨
131세, 935세도 같은 이유로 대체

Transaction Data¶

Merge 시 속성 조정¶

transaction data에 train의 gender Merge¶

m_transac_train에 songs data Merge¶

Support Vector Machines¶

Logistic Regression¶

k-Nearest Neighbor¶

gaussian classifiers¶

Decision tree¶

Artificial Neural Network¶

Evaluation¶

데이터 전처리¶

test에도 똑같이¶

변수 설명¶

quantitive variable¶

qualitative variable¶

dependent variable(y)¶

Support Vector Machines¶

Logistic Regression¶

k-Nearest Neighbor¶

gaussian classifiers¶

Decision tree¶

Artificial Neural Network¶

Evaluation¶

차원 축소(LDA)¶

	user_id	song_id	rec_loc	rec_screen	entry	listen
0	5	5	discover	Online playlist more	online-playlist	0
1	7	7	search	Search	song	1
2	14	14	discover	Online playlist more	song-based-playlist	0
3	22	22	my library	Local playlist more	local-library	0
4	24	24	search	Online playlist more	online-playlist	0

	all	suc	rec_rate
431	1564	1374	87.85
38	7085	5981	84.42
98	7254	5976	82.38
716	3485	2806	80.52
2	6400	5121	80.02
...	...	...	...
7867	71	11	15.49
3205	92	14	15.22
6549	86	13	15.12
9387	55	7	12.73
4975	76	8	10.53

	user_id	city	age	reg_method	reg_date	ex_date	gender
0	13827	14	29	9	20130112	20180203	female
1	6618	13	19	4	20160313	20180106	female
2	7615	22	33	9	20060314	20170930	female
3	15535	13	48	9	20110402	20171003	female
4	17708	18	20	4	20160102	20161105	female

	user_id	city	age	reg_method	reg_date	ex_date	gender	count
1802	13805	15	131	7	2010-12-16	2017-09-16	female	1
6767	1018	22	931	9	2011-10-17	2017-10-20	male	1

	age	length	count
count	1.019674e+06	1.019674e+06	1019674.0
mean	2.802562e+01	2.504527e+05	1.0
std	8.149761e+00	3.967728e+04	0.0
min	3.000000e+00	4.354600e+04	1.0
25%	2.300000e+01	2.236550e+05	1.0
50%	2.700000e+01	2.498350e+05	1.0
75%	3.100000e+01	2.756440e+05	1.0
max	1.050000e+02	7.584910e+05	1.0

	age	length
gender
female	28.190642	250020.416058
male	27.870786	250858.341855

	age	length	count
year_of_referance
1927	31.125000	289459.000000	1.0
1968	26.757009	275620.000000	1.0
1980	30.342857	310657.000000	1.0
1984	29.472222	302690.000000	1.0
1985	29.773333	263288.000000	1.0
1986	33.288235	198394.297059	1.0
1987	29.663793	255361.663793	1.0
1988	32.264798	263461.386293	1.0
1989	29.744565	262191.896739	1.0
1990	30.476596	282888.642553	1.0
1991	31.338798	299833.000000	1.0
1992	31.417112	269021.308378	1.0
1993	29.545263	306862.619649	1.0
1994	31.052738	286593.517241	1.0
1995	29.320628	282966.436099	1.0
1996	30.209119	282322.777778	1.0
1997	29.218176	278398.954411	1.0
1998	28.887542	269401.861170	1.0
1999	29.476810	257575.502085	1.0
2000	28.858168	262539.044146	1.0
2001	28.483621	265199.327038	1.0
2002	28.398612	260509.631904	1.0
2003	28.174489	255997.990587	1.0
2004	27.905534	265719.875234	1.0
2005	27.912200	255827.667221	1.0
2006	28.291326	257966.381006	1.0
2007	28.048811	255423.764290	1.0
2008	27.872759	261687.486866	1.0
2009	27.983808	257713.188898	1.0
2010	27.808410	257344.898107	1.0
2011	27.929324	260995.320671	1.0
2012	27.559251	249155.393687	1.0
2013	27.896090	258757.082813	1.0
2014	27.542313	251924.175455	1.0
2015	27.731263	246082.634961	1.0
2016	28.167318	242958.479015	1.0
2017	28.902467	241602.628416	1.0

	song_id	length	genre	artist	composer	lyricist	language	isrc	country_code	first_regist_code	year_of_referance	designation_code
1453	3744	247980	921	440	1267	1429	17	JPTK08680214	JP	TK0	1986	80214
1950	2393	237959	465	403	372	343	52	GBBBM8600001	GB	BBM	1986	00001
3704	8232	274974	465	403	372	343	52	GBBBM8600004	GB	BBM	1986	00004
3908	6210	147880	921	440	1267	1197	17	JPTK08680201	JP	TK0	1986	80201

<ml 교육 과제1> 남녀 성별 분류 모델¶

데이터 로딩 및 확인¶

transac_data¶

1) object로 형변환(user_id / song_id / listen)¶

songs_data¶

1) object로 형변환(song_id / genre / artist / composer / lyricist / language)¶

train_data¶

1) object로 형변환(city / reg_method)2) datetime으로 형변환(reg_date / ex_date)

test_data¶

1) object로 형변환(city / reg_method)2) datetime으로 형변환(reg_date / ex_date)

Training Data 탐색¶

age 이상값 대체(중앙값 27로)¶

만 0세는 있을 수 있으나, 나이 분포 고려했을 떄 이상값의 비율이 높을 것으로 예상됨 131세, 935세도 같은 이유로 대체

Transaction Data¶

Merge 시 속성 조정¶

transaction data에 train의 gender Merge¶

m_transac_train에 songs data Merge¶

Support Vector Machines¶

Logistic Regression¶

k-Nearest Neighbor¶

gaussian classifiers¶

Decision tree¶

Artificial Neural Network¶

Evaluation¶

데이터 전처리¶

test에도 똑같이¶

변수 설명¶

quantitive variable¶

qualitative variable¶

dependent variable(y)¶

Support Vector Machines¶

Logistic Regression¶

k-Nearest Neighbor¶

gaussian classifiers¶

Decision tree¶

Artificial Neural Network¶

Evaluation¶

차원 축소(LDA)¶

1) object로 형변환(city / reg_method)
2) datetime으로 형변환(reg_date / ex_date)

1) object로 형변환(city / reg_method)
2) datetime으로 형변환(reg_date / ex_date)

만 0세는 있을 수 있으나, 나이 분포 고려했을 떄 이상값의 비율이 높을 것으로 예상됨
131세, 935세도 같은 이유로 대체